var cheerio = require('cheerio');
var http = require('http');
var iconv = require('iconv-lite');
var fs = require('fs');
// 设置需要抓取的网页
var url = 'http://www.jobbole.com/';

http.get(url, function(sres) {
  var chunks = [];
  sres.on('data', function(chunk) {
    chunks.push(chunk);
  });
  sres.on('end', function() {
    // 定义一个临时存储数据的字段
    var result = {
      news: [],
      web: []
    };
    // 设置编码
    var html = iconv.decode(Buffer.concat(chunks), 'utf-8');
    // 使用类似jq的选择器
    var $ = cheerio.load(html, {decodeEntities: false});
    $('#widgets-homepage-fullwidth > div:nth-child(3) .grid-4').each((idx, element) => {
      var $element = $(element);
      $element.find('.floated-thumb').each(function(index, ele) {
        var url = $(ele).find('.post-thumb > a > img').attr('src');
        var title = $(ele).find('.post-meta .meta-title').text();
        result.news.push({
          imgUrl: url,
          title: title
        });
      });
    });  
    // 获取web数据
    $('#widgets-homepage-fullwidth > div:nth-child(7) > .floated-thumb').each((index, ele) => {
      var imgUrl = $(ele).find('.post-thumb > a > img').attr('src');
      var text = $(ele).find('.post-meta > p > a.meta-title').text();
      result.web.push({
        imgUrl: imgUrl,
        text: text
      });
    });
    // 把数据写入到本地
    fs.writeFile('./11.json', JSON.stringify(result, null, 2), { 'flag': 'w+' }, (error) => {
      if (error) {
        console.error('错误');
      }
      console.info('成功');
    });   
  });
});